Stable Diffusion#

Note

Install ekorpkit package first.

Set logging level to Warning, if you don’t want to see verbose logging.

If you run this notebook in Colab, set Hardware accelerator to GPU.

!pip install ekorpkit[art] exit()

Prepare Environment#

# %config InlineBackend.figure_format='retina'
%load_ext autotime
%load_ext autoreload
%autoreload 2 # reload modules before executing user code, set to 0 to disable

from ekorpkit import eKonf

eKonf.setLogger("WARNING")
eKonf.set_cuda(device=4)
print("version:", eKonf.__version__)

is_notebook = eKonf.is_notebook()
is_colab = eKonf.is_colab()
print("is notebook?", is_notebook)
print("is colab?", is_colab)
if is_colab:
    eKonf.mount_google_drive()
eKonf.set_workspace(workspace="/content/drive/MyDrive/workspace/", project="ekorpkit-book")

INFO:ekorpkit.base:Setting cuda device to ['A100-SXM4-40GB (id:4)']
INFO:ekorpkit.base:Setting EKORPKIT_WORKSPACE_ROOT to /content/drive/MyDrive/workspace/
INFO:ekorpkit.base:Setting EKORPKIT_PROJECT to ekorpkit-book
INFO:ekorpkit.base:Loaded .env from /workspace/projects/ekorpkit-book/config/.env
INFO:ekorpkit.base:Loaded .env from /workspace/projects/ekorpkit-book/config/.env

version: 0.1.40.post0.dev5
is notebook? True
is colab? False

('/content/drive/MyDrive/workspace', 'ekorpkit-book')

time: 3.4 s (started: 2022-11-03 10:55:32 +00:00)

Create a stable diffusion instance#

from ekorpkit.models.art.stable import StableDiffusion

# cfg = eKonf.compose("model/stable_diffusion")
# sd = eKonf.instantiate(cfg)
sd = StableDiffusion()

INFO:ekorpkit.base:Loaded .env from /workspace/projects/ekorpkit-book/config/.env

INFO:ekorpkit.base:Loaded .env from /workspace/projects/ekorpkit-book/config/.env

time: 58.1 s (started: 2022-11-03 10:55:36 +00:00)

Generate images#

text_prompts = "a living tiger inside a gorgeous cave in a futuristic Jeju island, relaxing night, watercolour on the ceiling, glowing on glass, light art, dark, dark, gorgeous, dark, bright in an abandoned village in a futuristic alien nightscape"
batch_name = "cave-jeju"

time: 714 µs (started: 2022-11-03 10:56:34 +00:00)

eKonf.setLogger("INFO")
sd.verbose = True

results = sd.imagine(
    text_prompts, 
    batch_name=batch_name, 
    num_samples=2, 
    guidance_scale=7
)

INFO:ekorpkit.batch:Using existing path: /content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen
INFO:ekorpkit.batch:Batch name: cave-jeju, Batch num: 0
INFO:ekorpkit.batch:Merging config with args: {}
INFO:ekorpkit.batch:Setting seed to 2749880475
INFO:ekorpkit.models.art.base:Prompt: a living tiger inside a gorgeous cave in a futuristic Jeju island, relaxing night, watercolour on the ceiling, glowing on glass, light art, dark, dark, gorgeous, dark, bright in an abandoned village in a futuristic alien nightscape
../../../_images/stable-diffusion_9_1.png 
INFO:ekorpkit.batch:Saving config to /content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/stable-diffusion/stable-batch/config/cave-jeju(0)_config.yaml

time: 33.2 s (started: 2022-11-03 10:56:34 +00:00)

Inpaint images#

base_url = "https://github.com/entelecheia/ekorpkit-book/raw/main/assets/figs"
init_image_path = f"{base_url}/chu-horse.png"
mask_image_path = f"{base_url}/chu-horse-masking-rough.png"
mask_image_exact_path = f"{base_url}/chu-horse-masking-exact.png"

init_image = eKonf.load_image(init_image_path)
mask_image = eKonf.load_image(mask_image_path)
mask_image_exact = eKonf.load_image(mask_image_exact_path)

sd.compare_images([mask_image, init_image, mask_image_exact], resize_ratio=0.5)
../../../_images/stable-diffusion_11_0.png 
time: 2.53 s (started: 2022-11-03 10:57:07 +00:00)

prompt = "a golden statue of an eagle with clouds, colorful painting by Tanya Hern, mural on the roof, digital art, artwork, beautiful, colorful, visual art on landscape, surrealism, watercolor, vivid by stunning"

results = sd.imagine(
    prompt, 
    mode="inpaint",
    init_image=init_image_path,
    mask_image=mask_image_path,
    num_samples=4,
    num_images_per_prompt=2, 
    inpaint_strength=10, 
    num_inference_steps=100,
    return_including_init_image=False
)

INFO:ekorpkit.batch:Using existing path: /content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen
INFO:ekorpkit.batch:Batch name: cave-jeju, Batch num: 1
INFO:ekorpkit.batch:Merging config with args: {}
INFO:ekorpkit.batch:Setting seed to 2776875379
INFO:ekorpkit.models.art.base:Prompt: a golden statue of an eagle with clouds, colorful painting by Tanya Hern, mural on the roof, digital art, artwork, beautiful, colorful, visual art on landscape, surrealism, watercolor, vivid by stunning
../../../_images/stable-diffusion_12_1.png 
INFO:ekorpkit.batch:Saving config to /content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/stable-diffusion/stable-batch/config/cave-jeju(1)_config.yaml

time: 54 s (started: 2022-11-03 10:57:10 +00:00)

Stitch images#

results = sd.imagine(
    text_prompts, 
    mode="stitch",
    batch_name=batch_name, 
    num_samples=6, 
    num_images_per_prompt=2,
    guidance_scale=7,
    inpaint_strength=10, 
)

INFO:ekorpkit.utils.notebook:shell type: ZMQInteractiveShell
../../../_images/stable-diffusion_14_1.png 
INFO:ekorpkit.models.art.stable: >> elapsed time to imagine: 0:02:49.117101
INFO:ekorpkit.batch:Saving config to /content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/stable-diffusion/stable-batch/config/cave-jeju(2)_config.yaml

time: 2min 49s (started: 2022-11-03 10:58:04 +00:00)

To view the panorama, upload the image to renderstuff.

Show Configuration#

sd.show_config(batch_name=batch_name, batch_num=0)

INFO:ekorpkit.batch:Using existing path: /content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen
INFO:ekorpkit.batch:Batch name: cave-jeju, Batch num: 0
INFO:ekorpkit.batch:Loading config from /content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/stable-diffusion/stable-batch/config/cave-jeju(0)_config.yaml
INFO:ekorpkit.batch:Merging config with the loaded config
INFO:ekorpkit.batch:Merging config with args: {}
INFO:ekorpkit.batch:Setting seed to 1894816122

{'_target_': 'ekorpkit.models.art.stable.StableDiffusion',
 'autoload': True,
 'batch': {'batch_name': 'cave-jeju',
           'batch_num': 0,
           'resume_run': False,
           'run_to_resume': 'latest',
           'seed': 1894816122,
           'set_seed': 'random_seed'},
 'collage': {'bg_color': 'black', 'cols': -1, 'max_images': 6, 'padding': 5},
 'device': 'cuda:7',
 'hf_user_access_token': None,
 'imagine': {'display_collage': True,
             'display_image': True,
             'guidance_scale': 7,
             'height': 512,
             'increase_seed_by': 1,
             'init_image': None,
             'inpaint_strength': 7.5,
             'make_collage': True,
             'mask_image': None,
             'mode': 'generate',
             'num_images_per_prompt': 1,
             'num_inference_steps': 100,
             'num_samples': 2,
             'save_collage': True,
             'save_image': True,
             'save_image_config': True,
             'seed': None,
             'text_prompts': 'a living tiger inside a gorgeous cave in a '
                             'futuristic Jeju island, relaxing night, '
                             'watercolour on the ceiling, glowing on glass, '
                             'light art, dark, dark, gorgeous, dark, bright in '
                             'an abandoned village in a futuristic alien '
                             'nightscape',
             'width': 768},
 'model': {'generate': {'device': 'cuda:6',
                        'pretrained_model_name_or_path': 'CompVis/stable-diffusion-v1-4',
                        'revision': 'fp16',
                        'use_auth_token': True},
           'inpaint': {'device': 'cuda:5',
                       'pretrained_model_name_or_path': 'runwayml/stable-diffusion-inpainting',
                       'revision': 'fp16',
                       'use_auth_token': True}},
 'module': {'library_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/libs'),
            'modules': None},
 'name': 'stable-diffusion',
 'path': {'batch': {'base_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/stable-diffusion/stable-batch'),
                    'name': 'stable-batch'},
          'batch_configs_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/stable-diffusion/batch_configs'),
          'batch_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/stable-diffusion/stable-batch'),
          'batch_name': 'stable-batch',
          'cache_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/cache'),
          'library_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/libs'),
          'model_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/models'),
          'name': 'disco-imagen',
          'output_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/outputs/stable-diffusion'),
          'root': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen'),
          'tmp_dir': PosixPath('/content/drive/MyDrive/workspace/projects/ekorpkit-book/disco-imagen/tmp'),
          'verbose': False},
 'verbose': False}
time: 44.2 ms (started: 2022-11-03 11:00:53 +00:00)